import time
import re
import collections
import math

"""
这个测试，准备将抓取到的nginx日志信息放到一个嵌套的字典中

"""

# Nginx的日志格式要求统一：
'''log_format  access_log_format  '"$remote_addr" "$remote_user" "[$time_local]" "$request" '
              '"$status" "$body_bytes_sent" "$bytes_sent" "$http_referer" '
              '"$http_user_agent" "$http_x_forwarded_for" "$http_host" "$request_time" "$upstream_response_time" "$upstream_addr" "$upstream_status"';'''

# 匹配Nginx日志使用统一的正则：
pattern_01 = r'''"(?P<remote_addr>[^"]+)" "(?P<remote_user>[^"]+)" "(?P<time_local>[^"]+)" "(?P<request>[^"]+)" "(?P<status>[^"]+)" "(?P<body_bytes_sent>[^"]+)" "(?P<bytes_sent>[^"]+)" "(?P<http_referer>[^"]+)" "(?P<http_user_agent>[^"]+)" "(?P<http_x_forwarded_for>[^"]+)" "(?P<http_host>[^"]+)" "(?P<request_time>[^"]+)" "(?P<upstream_response_time>[^"]+)" "(?P<upstream_addr>[^"]+)" "(?P<upstream_status>[^"]+)"'''

# 需要分析的日志文件路径：
log_file1 = "D:/k12日志分析测试/newk12_center.log_bak3"
# 定义字符串格式化
formatstring = '%-15s %-10s %-12s %8s %10s %10s %10s %10s %10s %10s %10s'


def convertBytes(bytes, lst=None):
    if lst is None:
        lst = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB']
    i = int(math.floor(  # 舍弃小数点，取小
        math.log(bytes, 1024)  # 求对数(对数：若 a**b = N 则 b 叫做以 a 为底 N 的对数)
    ))

    if i >= len(lst):
        i = len(lst) - 1
    return ('%.2f' + " " + lst[i]) % (bytes / math.pow(1024, i))


# 统计Nginx日志中各个IP的访问次数
def ip_count():
    # start = time.time()
    # 创建正则的regex对象
    regex = re.compile(pattern_01)
    # 创建一个嵌套字典
    ips = collections.defaultdict(dict)
    # 读取Nginx日志文件
    with open(log_file1, 'r', encoding='utf-8') as f:
        # 读取每行的日志
        for line in f:
            # 将每一行的日志转为字典
            mo = regex.search(line)
            if mo:
                # 获取日志中IP地址的信息
                ip_key = mo.group("remote_addr")

                if ips.get(ip_key) is None:
                    # 计算ip地址访问的次数times
                    ips[ip_key]['times'] = 1
                    # 计算ip地址产生的流量
                    ips[ip_key]['traffic'] = int(mo.group('bytes_sent'))
                else:
                    ips.get(ip_key)['times'] = ips.get(ip_key).get('times') + 1
                    # ips.get(ip_key)['bytes_sent'] = int(ips.get(ip_key).get('bytes_sent')) + int(mo.group('bytes_sent'))
                    ips.get(ip_key)['traffic'] = ips.get(ip_key).get('traffic') + int(mo.group('bytes_sent'))
    # print(ips)
    # 当ips字典整理后，进行排序，根据times排序
    c = sorted(ips, key=lambda k: ips[k]['times'], reverse=True)

    # 打印头信息
    # Total ip:
    print('Total IP: %s' % (len(ips)))
    # # formatstring = '%-15s %-10s %-12s %8s %10s %10s %10s %10s %10s %10s %10s'
    print('%-15s %-10s %-12s' % ('IP', '访问次数', '流量'))
    print('%-15s %-10s %-12s' % ('-' * 15, '-' * 10, '-' * 12))
    for ip in c:
        # 单位转换
        ip_traffic = convertBytes(ips.get(ip)['traffic'])
        print('%-15s %-10s %-12s' % (ip, ips.get(ip)['times'], ip_traffic))
    # end = time.time()
    # print('Running time: %.2f Seconds' % (end - start))


if __name__ == '__main__':
    ip_count()
